from scrapy.spiders import Spider
from scrapy.http import Request
from douban.items import DoubanItem
import re


class DoubanMovieSpider(Spider):
    name = 'movie'
    # custom_settings = {
    #     'ITEM_PIPELINES': {
    #         'douban.pipelines.DoubanPicPipelines': 1,
    #     }
    # }
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:54.0) Gecko/20100101 Firefox/54.0'
    }

    def start_requests(self):
        url = 'https://www.douban.com/doulist/1295618/'
        yield Request(url, headers=self.headers)

    def parse(self, response):
        item = DoubanItem()
        movies = response.xpath('//div[@class="doulist-item"]/div')
        for movie in movies:
            item['ranking'] = movie.xpath(
                './/span[@class="pos"]/text()').extract()[0]
            item['movie_name'] = movie.xpath(
                './/div[@class="title"]/a/text()').extract()[0]

            score = movie.xpath('.//span[@class="rating_nums"]/text()').extract()
            if score:
                score = score[0]
            else:
                score = 'N/A'
            item['score'] = score

            score_num = movie.xpath('.//div[@class="rating"]/span[3]').re(r'(\d+)')
            if score_num:
                score_num = score_num[0]
            else:
                score_num = 'N/A'
            item['score_num'] = score_num

            image_urls = movie.xpath('.//div[@class="post"]/a/img/@src').extract()
            # if image_urls:
            #   image_urls = image_urls[0]
            # else:
            #   image_urls = ''
            item['image_urls'] = image_urls

            item['images'] = movie.xpath(
                './/div[@class="post"]/a/img/@src').re(r'[^/]*.[jpg|png|gif|webp]$')

            yield item
        next_url = response.xpath('//span[@class="next"]/a/@href').extract()
        if next_url:
            yield Request(next_url[0], headers = self.headers)
